/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package app.scraper.slug;

/**
 *
 * @author Luis Javier
 */

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import java.io.IOException;
import java.util.logging.Level;
import java.util.logging.Logger;
import org.apache.commons.logging.LogFactory;
import org.w3c.css.sac.CSSException;
import org.w3c.css.sac.CSSParseException;
import org.w3c.css.sac.ErrorHandler;
import com.gargoylesoftware.htmlunit.javascript.JavaScriptErrorListener;
import com.gargoylesoftware.htmlunit.util.WebConnectionWrapper;

import java.net.MalformedURLException;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.IncorrectnessListener;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.ScriptException;
import com.gargoylesoftware.htmlunit.WebRequest;
import com.gargoylesoftware.htmlunit.WebResponse;
import com.gargoylesoftware.htmlunit.html.HTMLParserListener;
import com.gargoylesoftware.htmlunit.html.HtmlElement;
import java.net.URL;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import javax.swing.JComboBox;
import javax.swing.JOptionPane;
import javax.swing.JProgressBar;
import javax.swing.JTable;
import javax.swing.table.DefaultTableModel;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.select.Elements;
public class Scraper extends Thread{

    private WebClient web=new WebClient(BrowserVersion.FIREFOX_10);   
	public String link = "http://www.textyard.com/";
    HtmlPage page=null;
    public JComboBox school;
    public JTable depart;
    List<HtmlElement> schools=null;
    public JProgressBar bar;
    List<HtmlElement> depa=null;
    public JTable t_depar;
    public List<Integer> selection=null;
    public  JProgressBar bar2=null;
    public int school_index=0;
    public String file_name;
    
    
    @Override
    public void run() {
       load_page();
    }

    public Scraper() {
         LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log", "org.apache.commons.logging.impl.NoOpLog");

    java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit").setLevel(Level.OFF); 
    java.util.logging.Logger.getLogger("org.apache.commons.httpclient").setLevel(Level.OFF);

    
    web.setCssEnabled(false);

    web.setIncorrectnessListener(new IncorrectnessListener() {

        @Override
        public void notify(final String arg0, final Object arg1) {
            // TODO Auto-generated method stub

        }
    });
    web.setCssErrorHandler(new ErrorHandler() {

        @Override
        public void warning(final CSSParseException exception) throws CSSException {
            // TODO Auto-generated method stub

        }

        @Override
        public void fatalError(final CSSParseException exception) throws CSSException {
            // TODO Auto-generated method stub

        }

        @Override
        public void error(final CSSParseException exception) throws CSSException {
            // TODO Auto-generated method stub

        }
    });
    web.setJavaScriptErrorListener(new JavaScriptErrorListener() {

        @Override
        public void timeoutError(final HtmlPage arg0, final long arg1, final long arg2) {
            // TODO Auto-generated method stub

        }

        @Override
        public void scriptException(final HtmlPage arg0, final ScriptException arg1) {
            // TODO Auto-generated method stub

        }

        @Override
        public void malformedScriptURL(final HtmlPage arg0, final String arg1, final MalformedURLException arg2) {
            // TODO Auto-generated method stub

        }

        @Override
        public void loadScriptError(final HtmlPage arg0, final URL arg1, final Exception arg2) {
            // TODO Auto-generated method stub

        }
    });
    web.setHTMLParserListener(new HTMLParserListener() {

        @Override
        public void warning(final String arg0, final URL arg1, final int arg2, final int arg3, final String arg4) {
            // TODO Auto-generated method stub

        }

        @Override
        public void error(final String arg0, final URL arg1, final int arg2, final int arg3, final String arg4) {
            // TODO Auto-generated method stub

        }
    });
    web.setAjaxController(new NicelyResynchronizingAjaxController());
 //web.setJavaScriptEnabled(false);
    web.setThrowExceptionOnFailingStatusCode(false);
    web.setThrowExceptionOnScriptError(false);
    }
    
    
    
    
    public void load_page(){
        try {
			// bar.setStringPainted(true);
			// bar.setIndeterminate(true);
			// bar.setString("Loading Page");
            page=web.getPage(link);
			// web.getAjaxController().

			WebConnectionWrapper w = new WebConnectionWrapper(web) {
				@Override
				public WebResponse getResponse(final WebRequest request) throws IOException {
					WebResponse response = super.getResponse(request);
					System.out.println(response.getContentAsString());
					return response;
				}
			};
			w.getResponse(new WebRequest(
					new URL(
							"http://www.textyard.com/ajax/campus_ajax.php?q=a&limit=10&timestamp=1349985565870")));

			w.getResponse(new WebRequest(new URL(
					"http://www.textyard.com/ajax/dropdown_ajax.php?c=1105&term=68197&dept=363744")));
           // System.out.println(p.asText());
            //Document doc=Jsoup.parse(p.asXml());
           // Elements ele=doc.select("div[class=drop-list]").select("li");
            List<HtmlElement> list=(List<HtmlElement>) page.getByXPath("//div[@class='drop-list']");
            schools=list.get(0).getElementsByTagName("li");
            school.removeAllItems();
			// bar.setString("getting school list");
            for(int i=0; i<schools.size(); i++){
                System.out.println(schools.get(i).asXml());
               school.addItem(schools.get(i).asText());
               
              //  System.out.println("");
            }
			// bar.setIndeterminate(false);
			// bar.setString("Schools done.");
      
        } catch (IOException ex) {
            JOptionPane.showMessageDialog(null, "site Down\n"+ex.getMessage());
            Logger.getLogger(Scraper.class.getName()).log(Level.SEVERE, null, ex);
        } catch (FailingHttpStatusCodeException ex) {
            Logger.getLogger(Scraper.class.getName()).log(Level.SEVERE, null, ex);
        }
    }
    
   public void loadDeparment(final int index) {
        try {
            school_index=index;
            page=schools.get(index).click();
            web.waitForBackgroundJavaScript(1000);
            System.out.println(page.getElementById("school").asText());
            //department
            page.getElementById("department").mouseOver();
        //    System.out.println(page.getElementById("department").asXml());
            depa=page.getElementById("deptSelection").getElementsByTagName("li");
            DefaultTableModel model=(DefaultTableModel)depart.getModel();
            
            for(int i=0; i<depa.size(); i++){
               model.setNumRows(model.getRowCount()+1);
               model.setValueAt(depa.get(i).asText(), model.getRowCount()-1, 1);
               // System.out.println(depa.get(i).getAttribute("id"));
               
              //  System.out.println("");
            }
        //   page= depa.get(0).click();
          //  System.out.println(page.getElementById("courseSelection").asXml());
        } catch (IOException ex) {
            Logger.getLogger(Scraper.class.getName()).log(Level.SEVERE, null, ex);
        }
   
   }
           
 //---------------------------------------------------
   public void getCourses(){
     
       for(int i=0; i<selection.size(); i++){
          System.out.println("Selection: "+selection.get(i));          
            try {
                final String depa_id=depa.get(selection.get(i)).getAttribute("id").split("_")[1].trim();
                final List<String> courses=new ArrayList<String>();
                final List<String> courses_name=new ArrayList<String>();
                System.out.println(depa_id);
                page=depa.get(selection.get(i)).click();
                List<HtmlElement> li=page.getElementById("courseSelection").getElementsByTagName("li");
                 for(int j=0; j<li.size(); j++){
                     courses.add(li.get(j).getAttribute("id").split("_")[1].trim());
                     courses_name.add(li.get(j).asText());
                 }
                 scrape(depa_id, courses,courses_name,depa.get(selection.get(i)).asText(),schools.get(school_index).asText());
            } catch (Exception ex) {
                Logger.getLogger(Scraper.class.getName()).log(Level.SEVERE, null, ex);
            }
       }
       
       
   }
   
  //----------------------------------------------------
   public void scrape(final String depa_id,final List<String> courses, final List<String> courses_name,final String depa, final String school){
      Runnable r=new Runnable() {

            @Override
            public void run() {
                bar2.setIndeterminate(true);
                  bar2.setStringPainted(true);
               String info="http://www.slugbooks.com/acu/acu-Booklist.html?curDept="+depa_id+"&curCourse=CUR";
               WebClient client=new WebClient(BrowserVersion.FIREFOX_10);
               client.setThrowExceptionOnFailingStatusCode(false);
               client.setThrowExceptionOnScriptError(false);
               client.setAjaxController(new NicelyResynchronizingAjaxController());
               for(int i=0; i<courses.size(); i++){
                    try {
                        
                     HtmlPage p=   client.getPage(info.replace("CUR", courses.get(i)));
                    // client.waitForBackgroundJavaScript(2000);
                        //System.out.println("Cursos"+p.getElementById("course").asText());
                     
                     
                       
                       // Document doc=Jsoup.connect(info.replace("CUR", courses.get(i))).timeout(20000).get();
                      Document doc=Jsoup.parse(p.asXml());
                      Elements prices=doc.select("div[class=item-box]");
                        System.out.println("Prices: "+prices.size());
                        bar2.setString("Dept: "+depa+" Course: "+courses.get(i));
                        Elements a= doc.select("div[class=content-holder]").select("a");
                        int c=0;
                        for(int j=0; j<a.size(); j++){
                            if(a.get(j).text().contains("More Details")){
                               // p=client.getCurrentWindow().getWebClient().getPage(a.get(j).attr("abs:href"));
                               // client.waitForBackgroundJavaScript(2000);
                              //  Document doc1=Jsoup.parse(p.asXml());
                              Document doc1=Jsoup.connect(a.get(j).attr("abs:href")).timeout(20000).get();
                              Elements ele=doc1.select("div[class=info-section]");
                             String tit=ele.select("h1").text();
                             String aut=ele.select("span[class=author]").select("strong").text();
                             
                             Elements dt=doc1.select("div[class=info-section]").select("div[class=columns-box]").select("dt");
                              Elements dd=doc1.select("div[class=info-section]").select("div[class=columns-box]").select("dd");
                              String edition="",isbn13="",isbn="",plu="";
                             for(int z=0; z<dt.size(); z++){
                                 if(dt.get(z).text().contains("Edition"))
                                    edition=dd.get(z).text();
                                 else  if(dt.get(z).text().contains("ISBN13"))
                                     isbn13=dd.get(z).text();
                                 else  if(dt.get(z).text().contains("ISBN"))
                                     isbn=dd.get(z).text();
                                  else  if(dt.get(z).text().contains("Publisher"))
                                     plu=dd.get(z).text();
                             }
                             
                                          
                              Pattern pa=Pattern.compile("\\$\\d+\\.\\d+ New|\\$\\d+\\.\\d+ Used");
                       Matcher m=pa.matcher(prices.get(c).text());
                       String New="",Used="";
                       while(m.find()){
                        
                            String group=m.group();
                            if(m.group().contains("New"))
                                  New=group.replace("New", "").trim();
                            else if(m.group().contains("Used"))
                                  Used=group.replace("Used", "").trim();
                          
                            
                    
                       }           
                            
                             c++;
                                System.out.println("New: "+New);
                                System.out.println("Used: "+Used);
                                //System.out.println("Rent: "+prices.get(1).text());
                             
                                System.out.println("Title: "+tit);
                                System.out.println("Author: "+aut);
                                System.out.println("Edition: "+edition);
                                System.out.println("ISBN13: "+isbn13);
                                System.out.println("ISBN: "+isbn);
                                System.out.println("Publisher: "+plu);
                                System.out.println("Department: "+depa);
                                System.out.println("Department: "+school);
                                Results_Exporter rs=new Results_Exporter();
                                rs.file_name=file_name;
                                rs.export(tit, aut, isbn13, isbn, edition, depa,courses_name.get(i), school, New, Used);
                             System.out.println("");
                            }
                        }
                        
                    } catch (Exception ex) {
                        Logger.getLogger(Scraper.class.getName()).log(Level.SEVERE, null, ex);
                    } 
                    
               }
               bar2.setString("Done");
               bar2.setIndeterminate(false);
              client.closeAllWindows();
            }
        };
      Thread t=new Thread(r);
      t.start();
   }
    
}
